package wordstat;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

public class MetaGenerator {
    public static final String ENC = "UTF-8";
    private static final int TIMEOUT = 20000;
    public static final int DELAY_REQUEST = 12000;
    public static final int DELAY_PHRASE = 6000;
    public static final int MIN_WEIGHT = 10;

    //    public static final String GEO = "10174";
//    private static final String TEXT_GEO = "Санкт-Петербург+и+Ленинградская+область";
    public static final String GEO = "";
    private static final String TEXT_GEO = "";

//    public static final String BASE_PHRASES = "аккумулятор htc";
//    public static final String DATA_FILE = "data/accum.htc.txt";
//    public static final String BASE_PHRASES = "аккумулятор nokia";
//    public static final String DATA_FILE = "data/accum.nokia.txt";
//    public static final String BASE_PHRASES = "аккумулятор samsung";
//    public static final String DATA_FILE = "data/accum.samsung.txt";
//    public static final String BASE_PHRASES = "аккумулятор sony ericsson";
//    public static final String DATA_FILE = "data/accum.se.txt";
    public static final String BASE_PHRASES = "аккумулятор lg";
    public static final String DATA_FILE = "data/accum.lg.txt";

    //    public static final String BASE_PHRASES = "аккумулятор lg";
    public static final String STOP_PHRASES = "емкость,быстро,китайский,китайских,китайской,ноутбука,видеокамеры,фотоаппарата,зарядка,аккумулятора,разряжается,садится,замена,код,греется,активация,проверка,контакты,не,внешний,размеры,проверить,зарядное,устройство,калибровка,как,зарядить,двумя,снять,крышка,характеристики,mah";

    private static String[] basePhrases;
    private static String[] stopPhrases;

    static {
        basePhrases = BASE_PHRASES.split(",");
        stopPhrases = STOP_PHRASES.split(",");
    }

    public static void main(String[] args) throws IOException, InterruptedException {
        PrintStream metaWriter = new PrintStream(new FileOutputStream("meta/meta.txt"));
        List<Phrase> words = generate(basePhrases, stopPhrases);
        StringBuilder sb = new StringBuilder();
        for (Phrase word : words) {
            System.out.println(word.phrase + "=" + word.weight);
            sb.append(" ").append(word.phrase);
        }
        sb.deleteCharAt(0);
        metaWriter.print(sb);
        System.out.println(sb);

        List<String> list = checkPresent(new File(DATA_FILE), words);
        for (String word : list) {
            System.out.println(word);
        }
    }

    public static List<String> checkPresent(File dataFile, List<Phrase> words) throws IOException {
        List<String> list = new ArrayList<String>();
        // read entire file
        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
        InputStream is = new FileInputStream(dataFile);
        byte[] temp = new byte[1024];
        int read;
        while ((read = is.read(temp)) > 0) {
            buffer.write(temp, 0, read);
        }
        String data = new String(buffer.toByteArray()).toLowerCase();
        for (Phrase word : words) {
            if (!data.contains(word.phrase.toLowerCase())) {
                list.add(word.phrase);
            }
        }
        return list;
    }

    public static List<Phrase> generate(String[] phrases, String[] stopWords) throws InterruptedException, UnsupportedEncodingException, FileNotFoundException {
        String textGeoEnc = URLEncoder.encode(TEXT_GEO, ENC);
        int count = 0;
        List<String> uniqueWords = new ArrayList<String>();
        List<Phrase> words = new ArrayList<Phrase>();
        for (String phrase : phrases) {
            if (count++ != 0) {
                Thread.sleep(DELAY_REQUEST);
            }
            String URL = "http://wordstat.yandex.ru/?cmd=words";
            String text = URLEncoder.encode(phrase, ENC);
            int page = 1;
            while (true) {
                String readyURL = URL + "&page=" + page++ + "&text=" + text + "&geo=" + GEO + "&text_geo=" + textGeoEnc;
                System.out.println(readyURL);
                Document doc;
                try {
                    doc = Jsoup.connect(readyURL).
                            header("user-agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0").
                            header("Cookie", "yp=1628953306.sp.nd:50:ws:1:lang:!#1628139604.udn.c2hlbXBsaW5lcg==; yandexuid=36261312612900; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ; yabs-frequency=/3/hNO30ni9Pm000tUn01qR2Jm0005t0W0N60dS8t033HW9XPMj0mqO2Pm0/; my=YwECAAAnAgAALQECAA==; L=A39SC35HeVJlX05NeVpTUwMBAFh/bGdcNhoKMCoZYjE8UyAONRw9GgkaWCIdAnAeARQAOz8SMRpjBHdCJjNgWw==.1312779604.9074.268430.03ebff42cc506f579abb06a175673d47; spravka=dD0xMzEyNjQ5MjM4O2k9MTA5LjIwNS4yNTMuMTM1O3U9MTMxMjY0OTIzODkzNzY5MTU3MTtoPTY0M2FlZGZjYzkyZWQ2Mjk3OTIxOTUyYzhiYjk1Mjk2; t=p; yandex_login=shempliner; yandex_mail=shempliner; ys=udn.c2hlbXBsaW5lcg==; Session_id=1312779604.645.0.118550753.2:330838428:230.8:1312779604611:1388338972:13.77466.36850.267a73e88a26321cca687cc4f888684e").
                            timeout(TIMEOUT).
                            get();
                } catch (IOException e) {
                    Thread.sleep(DELAY_PHRASE);
                    System.out.println(e.getMessage());
                    continue; // reconnect
                }
                Elements rows = doc.select("tr.tlist");
                if (rows.size() == 0) {
                    break;
                }
                for (Element row : rows) {
                    String wordstatPhrase = row.child(0).text();
                    for (String stop : stopWords) {
                        if (wordstatPhrase.contains(stop)) {
                            continue;
                        }
                    }
                    String[] phraseFields = wordstatPhrase.split(" ");
                    int weight = Integer.parseInt(row.child(2).text());
                    if (weight < MIN_WEIGHT) {
                        break;
                    }
                    for (int i = 0; i < phraseFields.length - 1; i++) {
                        String field = phraseFields[i].replace("+", "");
                        if (!uniqueWords.contains(field)) {
                            uniqueWords.add(field);
                            Phrase word = new Phrase();
                            word.phrase = field;
                            word.weight = weight;
                            words.add(word);
                        }
                    }
                }
                Thread.sleep(DELAY_PHRASE);
            }
        }
        Collections.sort(words, new Comparator<Phrase>() {
            public int compare(Phrase p1, Phrase p2) {
                int result = 0;
                if (p1.weight > p2.weight) {
                    result = -1;
                } else if (p1.weight < p2.weight) {
                    result = 1;
                }
                return result;
            }
        });
        return words;
    }

    public static boolean containsStop(String phrase) {
        for (String stopPhrase : stopPhrases) {
            if (phrase.contains(stopPhrase)) {
                return true;
            }
        }
        return false;
    }

    static class Phrase {
        String phrase;
        int weight;
    }
}
